from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('127.0.0.1', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']


class POST():
    def __init__(self):
        self.params = {

        }
        self.data = {

        }
        self.headers = {

        }
        self.scrapy()

    def scrapy(self):
        r = requests.get('http://sa.offcn.com/zg/2018sxzp/')
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text)

        jd_list = soup.select('.zwxq') + soup.select('.zwxq2')

        for jd in jd_list:
            location = '北京'
            name = jd.find(class_ = 'zwname').text.strip()

            if len(jd.select('.skfx')) == 3:
                major = jd.find(class_ = 'skfx').get_text().strip()
            else:
                major = ''

            description = jd.find(text = re.compile('岗位职责')).parent.find_next_sibling().get_text() + jd.find(text = re.compile('任职资格')).parent.find_next_sibling().get_text()

            item = {
                "url": '',
                'edu': '',
                'exp': [],
                'name': name,
                'date': '',
                'lang': '',
                'place': '',
                'major': major,
                'count': -1,
                'salary': [],
                'toSchool': True,
                'welfare': [],
                'funType': '',
                'company': '北京中公教育科技股份有限公司',
                'location': location,
                'industry': '教育',
                'keywords': [],
                'platform': 'offical',
                'searchKeyword': '',
                'description': description,
                'subIndustry': '',
                'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            }

            result = offical_posts_coll.replace_one({'company': '北京中公教育科技股份有限公司', 'name': name, 'location': location}, item, True)
            print(item['company'], item['name'])


POST()
